import psycopg2
import gzip


DB_CONF = dict(database='crawl', user='bigdata', password='Youka888', host='192.168.31.102', port='5432')
SQL = """select title from jingdong_good_inf limit %s offset %s"""
LIMIT = 100


def get_wetao_data(conn_dict, out_file):
    conn = psycopg2.connect(**conn_dict)
    cur = conn.cursor()

    offset = 0

    with gzip.open(out_file, 'wb') as f:
        while True:
            cur.execute(SQL, (LIMIT, offset))
            offset += LIMIT
            rows = cur.fetchall()
            if len(rows) == 0:
                break
            print(rows)
            for row in rows:
                content = '\t'.join(['' if not r else r.replace('\r\n', ' ').replace('\n', ' ') for r in row]) + '\n'
                f.write(bytes(content, 'UTF-8'))
    conn.close()


if __name__ == '__main__':
    # import sys
    out = '/home/alpha/tmp/jd_good_titles.gz'
    get_wetao_data(DB_CONF, out)
